{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Compare Model Zoo Benchmark performance among different data types on Intel optimized  Tensorflow\n",
    "\n",
    "This jupyter notebook will help you evaluate performance benefits among different data types like int8/bf16 on Intel-optimized Tensorflow via several models from Intel Model Zoo. \n",
    "The notebook will show users some bar charts like below for performance comparison among different data types."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images\\perf_comparison_types.png\"  />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Section 1: Display Platform Information "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignore all warning messages\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ACTION: Users should change the value of os.environ['ModelZooRoot'] according to their environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# If default path does not work, change ModelZooRoot path according to your environment\n",
    "## USER INPUT\n",
    "accuracy_only=False\n",
    "current_path = os.getcwd()\n",
    "os.environ['ModelZooRoot'] = current_path + \"/../../../\"\n",
    "os.environ['ProfileUtilsRoot'] = os.environ['ModelZooRoot'] + \"docs/notebooks/perf_analysis/profiling/\"\n",
    "print(os.environ['ModelZooRoot'])\n",
    "print(os.environ['ProfileUtilsRoot'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Check for mandatory python scripts after ModelZooRoot and ProfileUtilsRoot are assigned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "current_path = os.getcwd()\n",
    "benchmark_path = os.environ['ModelZooRoot'] + \"benchmarks/launch_benchmark.py\"\n",
    "if os.path.exists(benchmark_path) == True:\n",
    "    print(benchmark_path)\n",
    "else:\n",
    "    print(\"ERROR! Can't find benchmark script!\")\n",
    "    \n",
    "profile_utils_path = os.environ['ProfileUtilsRoot'] + \"profile_utils.py\"\n",
    "if os.path.exists(profile_utils_path) == True:\n",
    "    print(profile_utils_path)\n",
    "else:\n",
    "    print(\"ERROR! Can't find profile_utils script!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from profiling.profile_utils import PlatformUtils\n",
    "plat_utils = PlatformUtils()\n",
    "plat_utils.dump_platform_info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Section 2: Run the benchmark on the Intel TensorFlow "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Check TensorFlow version and oneDNN enablement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "print (\"We are using Tensorflow version\", tf.__version__)\n",
    "major_version = int(tf.__version__.split(\".\")[0])\n",
    "if major_version >= 2:\n",
    "   from tensorflow.python.util import _pywrap_util_port\n",
    "   print(\"oneDNN enabled:\", _pywrap_util_port.IsMklEnabled())\n",
    "else:\n",
    "   print(\"oneDNN enabled:\", tf.pywrap_tensorflow.IsMklEnabled())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_2'></a>\n",
    "## Step 2 : Select a supported Topology/Benchmark with different data types support\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_2_1'></a>\n",
    "### Step 2.1 : List out the Topologies/Benchmarks with multiple data types support"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Set accuracy_only to True if needed\n",
    "By default, we benchmark the topology for throughput only. Please set accuracy_only to True if you also want to get accuracy number. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_only=False\n",
    "# overwrite the accuracy_only value by the \"ACCURACY_ENABLE\" environment variable.\n",
    "import os\n",
    "accuracy_enable=os.environ.get('ACCURACY_ENABLE', '')\n",
    "if accuracy_enable != '':\n",
    "    accuracy_only= eval(accuracy_enable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List out all supported benchmarks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from profiling.profile_utils import ConfigFile, CSVCommonUtils\n",
    "config = ConfigFile()\n",
    "df, df_types, df_types_obj = config.convert_configs_to_pd_dataframe(accuracy_only=accuracy_only)\n",
    "\n",
    "name_list = []\n",
    "precision_list = []\n",
    "for name,group in df_types_obj:\n",
    "    name_list.append(name)\n",
    "\n",
    "cache_csv_file = \"cache.csv\"\n",
    "fnames = ['csv_fname_list', 'selected_precision_list']\n",
    "if os.path.isfile(cache_csv_file) is False:#only initialize list for the first run among data types\n",
    "    # Initial CSV files list\n",
    "    csv_fname_list = []\n",
    "    selected_precision_list=[]\n",
    "    cachecsv = CSVCommonUtils(fnames, cache_csv_file)\n",
    "else:\n",
    "    cachecsv = CSVCommonUtils(fnames, cache_csv_file)\n",
    "    csv_fname_list, selected_precision_list = cachecsv.read_from_csv()\n",
    "\n",
    "print(csv_fname_list)\n",
    "print(selected_precision_list)\n",
    "    \n",
    "print(\" Benchmark that supports multiple data types :\")\n",
    "index=0\n",
    "for name in name_list:\n",
    "    print(' Index %d : %s' %(index, name))\n",
    "    index +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_2_2'></a>\n",
    "### Step 2.2 : Pick a Topology/Benchmark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The User needs to pick a topology for the first run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User picks a benchmark type\n",
    "# use the \"TOPO_INDEX\" environment variable value if it exists.\n",
    "import os\n",
    "env_topo_index=os.environ.get('TOPO_INDEX', '')\n",
    "if env_topo_index != '':\n",
    "    selected_index= int(env_topo_index)\n",
    "else:\n",
    "    ## USER INPUT\n",
    "    selected_index= int(input('Input a index number of a topology: '))\n",
    "os.environ['TOPO_INDEX']=str(selected_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "show the selected topology."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if selected_index > index:\n",
    "    print(\"Please select a valid index number.\")\n",
    "else:\n",
    "    selected_df = df_types_obj.get_group(name_list[selected_index])\n",
    "    #print(name_list[selected_index])\n",
    "    selected_topology = name_list[selected_index][0] +'-'+name_list[selected_index][1]\n",
    "    print(selected_topology)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_3'></a>\n",
    "## Step 3 : Select a supported data types for the Topology/Benchmark\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_3_1'></a>\n",
    "### Step 3.1 : List out all data types supported by the selected Topology/Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_df = df_types_obj.get_group(name_list[selected_index])\n",
    "print(name_list[selected_index])\n",
    "print(\" supported data types :\")\n",
    "precision_df = selected_df[\"precision\"]\n",
    "\n",
    "for i in range(len(precision_df)):\n",
    "    print(' Index %d : %s' %(i, selected_df[\"precision\"].iloc[i]))\n",
    "    precision_list.append(selected_df[\"precision\"].iloc[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_3_2'></a>\n",
    "### Step 3.2 : Pick a Data Type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ACTION : Please select one supported data type and change data_type_index accordingly\n",
    "By default, users can start with 0 as data_type_index."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User picks a data type for selected benchmark type\n",
    "if len(csv_fname_list) == 0: #only for the first run among data types\n",
    "    data_type_index = 0\n",
    "# overwrite the accuracy_only value by the \"ACCURACY_ENABLE\" environment variable.\n",
    "env_data_type_index=os.environ.get('DATA_TYPE_INDEX', '')\n",
    "if env_data_type_index != '':\n",
    "    data_type_index= int(env_data_type_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_3_3'></a>\n",
    "### Step 3.3 : List out the selected topology/benchmark name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if data_type_index >= len(precision_df):\n",
    "    print(\"Please select a valid index number.\")\n",
    "else:\n",
    "    topology_name = selected_df.iloc[data_type_index]['benchmark']\n",
    "    print(\"selected topology/benchmark for this run : \", topology_name)\n",
    "\n",
    "selected_precision_list.append(precision_list[data_type_index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Export the selected topology and data type as environment variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['SELECTED_TOPO'] = selected_topology\n",
    "os.environ['SELECTED_TYPE'] = selected_df[\"precision\"].iloc[data_type_index]\n",
    "print(os.environ['SELECTED_TOPO'])\n",
    "print(os.environ['SELECTED_TYPE'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_4'></a>\n",
    "## Step 4: Configure parameters for launch_benchmark.py according to the selected Topology"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4.1: Import Model Zoo CPU info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append(os.environ['ModelZooRoot']+os.sep+'benchmarks/common/')\n",
    "from platform_util import PlatformUtil \n",
    "cpu_info = PlatformUtil(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4.2: User can also manually set batch size and number of threads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "numa_nodes = cpu_info.numa_nodes\n",
    "print(\"CPU count per socket:\" , cpu_info.cores_per_socket ,\" \\nSocket count:\", cpu_info.sockets, \" \\nNuma nodes:\",numa_nodes)\n",
    "if numa_nodes > 0:\n",
    "    socket_number = 1\n",
    "    cpu_count = cpu_info.cores_per_socket\n",
    "    inter_thread = 1\n",
    "else:\n",
    "    # on non-numa machine, we should use all the cores and don't use numactl\n",
    "    socket_number = -1\n",
    "    cpu_count = cpu_info.cores_per_socket * cpu_info.sockets\n",
    "    inter_thread = cpu_info.sockets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ACTION: Users can change the value of thread_number and batch_size to see different performance\n",
    "1. thread_umber: the value will apply to num_cores parameters in launch_benchmark.py  \n",
    "2. utilized_socket_number:  the value will apply to the socket-id parameter in launch_benchmark.py \n",
    "3. num_inter_threads: the value will  apply to the num-inter-threads parameter in launch_benchmark.py \n",
    "4. num_intra_threads: the value will  apply to the num-intra-threads parameter in launch_benchmark.py \n",
    "5. batch_size: the value will apply to the batch_size parameter in launch_benchmark.py \n",
    "6. log_folder: the folder where the logs are stored.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Here are some performance optimzation BKMs for your reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the parameters from config\n",
    "config = ConfigFile()\n",
    "config.read_config(topology_name)\n",
    "if config.perf_bkm != '':\n",
    "    print(config.perf_bkm)\n",
    "else:\n",
    "    print(\"No BKM for this topology\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## USER INPUT\n",
    "thread_number=cpu_count \n",
    "utilized_socket_number=socket_number\n",
    "num_inter_threads = inter_thread\n",
    "num_intra_threads = thread_number\n",
    "batch_size=config.batch_size # Used recommended batch_size if any. Users can overwrite the value of batch_size\n",
    "log_folder=os.getcwd() + os.sep + \"logs\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4.3: get required data and files if needed.\n",
    "No action if there is no output from this below cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optional ACTION: Users input the folder paths for those required data and files\n",
    "1. data_download_path: the value will be set as data-location parameters in topo.ini for the related topology.\n",
    "2. model_source_dir: the value will be set as the model-source-dir parameter in topo.ini for the related topology such as Wide and Deep. Those models use tensorflow-models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# Get the parameters from config\n",
    "config = ConfigFile()\n",
    "config.read_config(topology_name)\n",
    "data_download_path=''\n",
    "model_source_dir=''\n",
    "if ( config.data_download != '' or (config.data_download_accuracy != '' and accuracy_only is True) ) and config.data_location == '':\n",
    "    print(\"\\nPlease follow below command to get the data : \")\n",
    "    val = config.read_value_from_section(topology_name, 'data-download') if config.data_download != '' else config.read_value_from_section(topology_name, 'data-download-accuracy')\n",
    "    print(val)\n",
    "    # use the \"DATA_DOWNLOAD_PATH\" environment variable value if it exists.\n",
    "    env_data_download_path=os.environ.get('DATA_DOWNLOAD_PATH', '')\n",
    "    if env_data_download_path != '':\n",
    "        data_download_path= env_data_download_path\n",
    "    else:\n",
    "        ## USER INPUT\n",
    "        data_download_path= input('Input a data download path: ')\n",
    "    print(\" data download path : \" , data_download_path)\n",
    "\n",
    "if config.preprocessing != '':\n",
    "    print(\"\\nPlease follow below command to get required files and installation : \")\n",
    "    val = config.read_value_from_section(topology_name, 'preprocessing')\n",
    "    print(val)\n",
    "    # use the \"MODEL_SOURCE_DIR\" environment variable value if it exists.\n",
    "    env_model_source_dir=os.environ.get('MODEL_SOURCE_DIR', '')\n",
    "    if env_model_source_dir != '':\n",
    "        model_source_dir= env_model_source_dir\n",
    "    else:\n",
    "        ## USER INPUT\n",
    "        model_source_dir= input('Input a model source dir: ')\n",
    "    print(\" model_source_dir : \" , model_source_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">NOTE: If no action required from above cell, please skip below cell and go to [Step 4.4](#step_4_4)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Users set the configurations in topo.ini for those required data and files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Overwrite configurations in topo.ini\n",
    "config = ConfigFile()\n",
    "config.read_config(topology_name)\n",
    "\n",
    "if ( config.data_download != '' or (config.data_download_accuracy != '' and accuracy_only is True) ) and data_download_path != '':\n",
    "    config.write_value_from_section(topology_name, 'data-location', data_download_path)\n",
    "    config.data_location = data_download_path\n",
    "\n",
    "if accuracy_only == True and config.data_location == '':\n",
    "    print(\"ERROR! STOP! need data for accuacy evaluatoin!\")  \n",
    "\n",
    "if config.preprocessing != '':\n",
    "    config.write_value_from_section(topology_name, 'model-source-dir', model_source_dir)    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_4_4'></a>\n",
    "### Step 4.4: Prepare the pre-trained model and model parameters for running the benchmark\n",
    "1. Get related parameters according to the selected topology\n",
    "2. Get the pretrained model if needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "config = ConfigFile()\n",
    "configvals = []\n",
    "# Get common parameters according to users' inputs  \n",
    "params = config.get_parameters(topology_name, configvals,\n",
    "                   batch_size=batch_size, thread_number=thread_number, socket_number=utilized_socket_number,\n",
    "                   num_inter_threads=num_inter_threads, num_intra_threads=num_intra_threads, accuracy_only=accuracy_only)\n",
    "\n",
    "# Get the parameters from config\n",
    "configvals=config.read_config(topology_name)\n",
    "\n",
    "# Get the pre-trained model file\n",
    "if config.wget != '' and ( config.in_graph == '' or config.checkpoint == ''  ):\n",
    "    pretrain_model_path = config.download_pretrained_model(current_path=current_path)\n",
    "    pretrain_model_path = config.uncompress_file(pretrain_model_path, current_path=current_path)\n",
    "    if config.in_graph == 'NA':\n",
    "        config.checkpoint = pretrain_model_path        \n",
    "    if config.checkpoint == 'NA':\n",
    "        pretrain_model_path = config.find_pretrained_model_in_folder(pretrain_model_path)\n",
    "        config.in_graph = pretrain_model_path \n",
    "# set pre-trained model path        \n",
    "if config.checkpoint == 'NA':\n",
    "    configvals.append(\"--in-graph\")\n",
    "    configvals.append(config.in_graph)\n",
    "if config.in_graph == 'NA':\n",
    "    configvals.append(\"--checkpoint\")\n",
    "    configvals.append(config.checkpoint)\n",
    "    \n",
    "#Set output-dir folder\n",
    "if log_folder !='':\n",
    "    configvals.append(\"--output-dir\")\n",
    "    configvals.append(log_folder)\n",
    "\n",
    "# Add custom arguments\n",
    "if config.custom_args != '':\n",
    "    configvals.append(\"--\")\n",
    "    custom_config = config.parsing_custom_args(topology_name, config.custom_args)\n",
    "    configvals = configvals + custom_config\n",
    "\n",
    "# Combine common parameters and config parameters\n",
    "params = params + configvals    \n",
    "    \n",
    "sys.argv=[benchmark_path]+params\n",
    "print(sys.argv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4.5: Create a CSV file to log the performance numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from profiling.profile_utils import PerfPresenter\n",
    "job_type = topology_name.split(' ')[1]\n",
    "csv_fname=job_type+'_'+topology_name.replace(' ', '')+'.csv'\n",
    "print(csv_fname)\n",
    "perfp=PerfPresenter()\n",
    "perfp.create_csv_logfile(job_type, csv_fname)\n",
    "\n",
    "found = False\n",
    "for csv in csv_fname_list:\n",
    "    if csv == csv_fname:\n",
    "        found = True\n",
    "        break\n",
    "if found == False:\n",
    "    csv_fname_list.append(csv_fname)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4.6: Retrieve oneDNN Runtime Information\n",
    "> NOTE : performance may be impacted if users enable those oneDNN debug features."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Users can enable oneDNN VERBOSE Mode to have more information from oneDNN library.  \n",
    "Users can export the DNNL_VERBOSE environment variable to turn verbose mode on and control the level of verbosity.\n",
    "\n",
    "|Environment variable|Value|Description|\n",
    "|:-----|:----|:-----|\n",
    "|DNNL_VERBOSE| 0 |no verbose output (default)|\n",
    "||1|primitive information at execution|\n",
    "||2|primitive information at creation and execution|  \n",
    "\n",
    "Refer to the [link](https://oneapi-src.github.io/oneDNN/dev_guide_verbose.html) for detailed verbose mode information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['DNNL_VERBOSE'] = '1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Users can use the DNNL_JIT_DUMP environment variable to inspect oneDNN JIT code,  \n",
    "and then check instructions usage by dissassembling the JIT kernel.\n",
    "\n",
    "|Environment variable|Value|Description|\n",
    "|:-----|:----|:-----|\n",
    "|DNNL_JIT_DUMP | 0 |JIT dump is disabled (default)|\n",
    "||any other value|JIT dump is enabled|\n",
    "\n",
    "Refer to the [link](https://oneapi-src.github.io/oneDNN/dev_guide_inspecting_jit.html) for detailed JIT Dump information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> NOTE: recomend only enable JIT DUMP for inference. For training, number of JIT DUMP files would be huge!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['DNNL_JIT_DUMP'] = '0'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_5'></a>\n",
    "## Step 5:  Run the benchmark "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> NOTE: Below section will enable Tensorflow timeline for the model by patching it, and then unpatch it after the model completes its training or inference."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set enable_tf_timeline to False if users don't want to get TF timeline information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "enable_tf_timeline = True\n",
    "# overwrite the accuracy_only value by the \"ENABLE_TIMELINE\" environment variable.\n",
    "timeline_enable=os.environ.get('ENABLE_TIMELINE', '')\n",
    "if timeline_enable != '':\n",
    "    enable_tf_timeline= eval(timeline_enable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the benchmark\n",
    "> NOTE: Users don't need to finish training if whole training takes a long time.  \n",
    "Users can stop below cell in the middle of the training, and still get the related performance data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if enable_tf_timeline == True:\n",
    "    # patch related model script\n",
    "    repo_path = os.environ['ModelZooRoot'] #current_path + os.sep + \"../../\"\n",
    "    config.patch_model_to_enable_timeline(repopath=repo_path)\n",
    "\n",
    "# run the benchmark with the patch\n",
    "import sys\n",
    "benchmark_path = os.environ['ModelZooRoot']+os.sep+\"benchmarks/\"\n",
    "sys.path.append(benchmark_path)\n",
    "from launch_benchmark import LaunchBenchmark\n",
    "\n",
    "util = LaunchBenchmark()\n",
    "util.main()\n",
    "\n",
    "if enable_tf_timeline == True:\n",
    "    # unpatch related model script\n",
    "    config.unpatch_model_to_enable_timeline(model_path=repo_path+'/models/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_6'></a>\n",
    "## Step 6: Parse output for performance number"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Found the file path of the related runtime log."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# identify the path of the latest log file\n",
    "configvals=config.read_config(topology_name)\n",
    "import os\n",
    "for file in os.listdir(log_folder):\n",
    "    if file.endswith(\".log\"):\n",
    "        logpath = os.path.join(log_folder, file)\n",
    "        used_logpath = logpath + \".old\"\n",
    "        os.rename(logpath, used_logpath)\n",
    "        print(used_logpath)\n",
    "        break\n",
    "os.environ[\"TF_LOGPATH\"] = used_logpath"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parse the logfile for performance number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if accuracy_only == False:\n",
    "    print(\"get throughput\")\n",
    "    val = config.throughput_keyword\n",
    "    index = int(config.throughput_index)\n",
    "    splitter = config.throughput_splitter\n",
    "    line = perfp.read_throughput(used_logpath, keyword=val, index=index, splitter=splitter)\n",
    "    if line!=None:\n",
    "        throughput=line\n",
    "        print(throughput)\n",
    "        # log the perf number\n",
    "        perfp.log_infer_perfcsv(0, throughput, 0, csv_fname)\n",
    "    else:\n",
    "        print(\"ERROR! can't find correct performance number from log. please check log for runtime issues\")\n",
    "else:\n",
    "    # get accuracy number and caculate throughput\n",
    "    print(\"get accuracy and throughput\")\n",
    "    #val = config.throughput_keyword\n",
    "    #index = int(config.throughput_index)\n",
    "    accuracy = perfp.read_accuracy(used_logpath)\n",
    "    iternation = perfp.read_iteration_time(used_logpath)\n",
    "    if accuracy != [] and iternation != []:\n",
    "        final_accuracy=accuracy[-1]\n",
    "        iternation_time = iternation[-1]\n",
    "        throughput = float(batch_size)/iternation_time\n",
    "        print(final_accuracy,throughput)\n",
    "        # log the perf number\n",
    "        perfp.log_infer_perfcsv(0, throughput, final_accuracy, csv_fname)\n",
    "    else:\n",
    "        print(\"ERROR! can't find correct performance number from log. please check log for runtime issues\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optional : print out the log file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_out_log = False\n",
    "if print_out_log is True:\n",
    "    logfile = open(used_logpath)\n",
    "    logout = logfile.read()\n",
    "    print(logout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Users should be able to see a new Timeline json file after running the benchmark\n",
    "If users don't see a new timeline json file, they need to make sure that they patch the model script correctly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from profiling.profile_utils import CommonUtils\n",
    "utils = CommonUtils()\n",
    "paths = []\n",
    "paths.append(os.environ['ModelZooRoot']+os.sep + \"benchmarks\")\n",
    "paths.append(os.environ['ModelZooRoot']+os.sep + \"docs/notebooks/perf_analysis\")\n",
    "pattern = \"*.json\"\n",
    "timeline_files, timeline_paths = utils.found_files_in_folders(pattern, paths)\n",
    "if timeline_paths == []:\n",
    "    print(\"No %s files found\" %(pattern))\n",
    "else:\n",
    "    print(timeline_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Users should be able to see new JIT DUMP files after running the benchmark if they enable the oneDNN JIT DUMP feature.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from profiling.profile_utils import CommonUtils\n",
    "utils = CommonUtils()\n",
    "paths = []\n",
    "paths.append(os.environ['ModelZooRoot']+os.sep + \"benchmarks\")\n",
    "paths.append(os.environ['ModelZooRoot']+os.sep + \"docs/notebooks/perf_analysis\")\n",
    "pattern = \"*.bin\"\n",
    "jitdump_files, jitdump_paths = utils.found_files_in_folders(pattern, paths)\n",
    "if jitdump_paths == []:\n",
    "    print(\"No %s files found\" %(pattern) ,\"Need to set DNNL_JIT_DUMP as 1 first\")\n",
    "else:\n",
    "    print(jitdump_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Gather all generated JIT DUMP files\n",
    "Copy the jit dump files from benchmark folder to the JITDUMP folder.  \n",
    "Those jit dump files will be analyzed in another Jupyter notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "\n",
    "jitdump_dir_path = os.environ['ModelZooRoot']+os.sep + \"docs/notebooks/perf_analysis\" + os.sep + \"JITDUMP\"\n",
    "if os.path.isfile(jitdump_dir_path) == False:\n",
    "    os.mkdir(jitdump_dir_path)\n",
    "shutil.move(os.environ['TF_LOGPATH'],jitdump_dir_path)\n",
    "if jitdump_paths != []:\n",
    "    for path in jitdump_paths:\n",
    "        shutil.move(path,jitdump_dir_path)\n",
    "target_path = jitdump_dir_path+'_'+os.environ['SELECTED_TYPE']\n",
    "if os.path.isfile(target_path) == True:\n",
    "    os.remove(target_path)\n",
    "shutil.move(jitdump_dir_path, target_path)\n",
    "print(target_path)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Repeat Step 3 to Step 6 among different data types for selected topology/benchmark\n",
    "> NOTE : Please iterate over different data types before you start Step 7 for performance comparison.\n",
    "Users can pick one of below options."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = [csv_fname_list[-1], selected_precision_list[-1]]\n",
    "cachecsv.write_to_csv(fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Option 1: Automatically pick next data type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_type_index +=1\n",
    "if data_type_index < len(precision_list):\n",
    "    os.environ['DATA_TYPE_INDEX'] = str(data_type_index)\n",
    "else:\n",
    "    os.environ['DATA_TYPE_INDEX'] = ''\n",
    "    os.environ['TOPO_INDEX'] = ''\n",
    "    cachecsv.delete_csv()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Go back Step 3.3 by clicking the link : [Step 3.3](#step_3_3) or run below cell to automatically go over all steps for next date type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Option 2: manully pick next data type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Go back Step 3.2 by clicking the link : [Step 3.2](#step_3_2)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_7'></a>\n",
    "## Step 7: Draw the performance comparison diagram\n",
    "> NOTE: Please iterate over different data types before the Step 7 and the Step 8\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for csv in csv_fname_list:\n",
    "    print(csv)\n",
    "for precision in selected_precision_list:\n",
    "    print(precision)\n",
    "print(selected_topology)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from profiling.profile_utils import PerfPresenter\n",
    "\n",
    "perfp=PerfPresenter(True)\n",
    "# inference  throughput\n",
    "title = 'Perf comparison among data types'\n",
    "perfp.draw_perf_diag_from_csvs(csv_fname_list, selected_precision_list, 'throughput','throughput (image/sec)', selected_topology, title)\n",
    "perfp.draw_perf_ratio_diag_from_csvs(csv_fname_list, selected_precision_list, 'throughput','speedup', selected_topology, title)\n",
    "if accuracy_only == True:\n",
    "    perfp.draw_perf_diag_from_csvs(csv_fname_list, selected_precision_list,'accuracy','accuracy', selected_topology, title)\n",
    "    perfp.draw_perf_ratio_diag_from_csvs(csv_fname_list, selected_precision_list,'accuracy','accuracy loss', selected_topology, title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='step_8'></a>\n",
    "## Step 8: Gather all generated Tensorflow Timeline Json files and JITDUMP files\n",
    "Copy the timeline json files from benchmark folder to the Timeline folder.\n",
    "Those Timeline files will be analyzed in another Jupyter notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "import datetime\n",
    "from profiling.profile_utils import CommonUtils\n",
    "utils = CommonUtils()\n",
    "print(len(precision_list))\n",
    "if timeline_paths != [] and len(timeline_paths) >= len(precision_list):\n",
    "    timeline_dir_path = os.environ['ModelZooRoot']+os.sep + \"docs/notebooks/perf_analysis\" + os.sep + \"Timeline\"\n",
    "    if os.path.isfile(timeline_dir_path) == False:\n",
    "        os.mkdir(timeline_dir_path)\n",
    "    for path in timeline_paths:\n",
    "        shutil.move(path,timeline_dir_path)\n",
    "    # move JITDUMP results into Timeline folder     \n",
    "    pattern = \"JITDUMP_*\"\n",
    "    jitdump_fds, jitdump_fd_paths = utils.found_files_in_folder(pattern, os.environ['ModelZooRoot']+os.sep + \"docs/notebooks/perf_analysis\")\n",
    "    for fd_path in jitdump_fd_paths:\n",
    "        print(fd_path)\n",
    "        shutil.move(fd_path,timeline_dir_path)\n",
    "    # rename Timeline folder with topo and time info   \n",
    "    timeinfo = datetime.datetime.now().strftime(\"%Y-%m-%d_%H:%M\")\n",
    "    target_path = timeline_dir_path+'_'+os.environ['SELECTED_TOPO']+'_'+timeinfo\n",
    "    shutil.move(timeline_dir_path, target_path)\n",
    "    cachecsv.delete_csv()\n",
    "    print(\"Successfully gather all results in \",target_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "intel-tensorflow",
   "language": "python",
   "name": "intel-tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
